{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# some_file.py\n",
    "import sys\n",
    "# insert at 1, 0 is the script path (or '' in REPL)\n",
    "sys.path.insert(1, '../')\n",
    "from lxrt.SlowFast.slowfast.config.defaults import get_cfg\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lxrt.SlowFast.slowfast.datasets.tgif_direct import TGIF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = TGIF(cfg, \"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "cfg = get_cfg()\n",
    "cfg_file = \"../lxrt/SlowFast/configs/Kinetics/c2/SLOWFAST_8x8_R50.yaml\"\n",
    "cfg.merge_from_file(cfg_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/users/cdwivedi/RL_EXP/IDL/project/tgif-qa/code/dataset/tgif/gifs/tumblr_nqc2mbmU2J1uxhtnwo1_400.gi/*\n",
      "torch.Size([3, 8, 256, 256]) torch.Size([3, 32, 256, 256])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[tensor([[[[-1.9914, -1.9918, -1.9923,  ..., -1.9882, -1.9883, -1.9884],\n",
       "           [-1.9917, -1.9920, -1.9924,  ..., -1.9884, -1.9885, -1.9886],\n",
       "           [-1.9919, -1.9921, -1.9924,  ..., -1.9886, -1.9888, -1.9889],\n",
       "           ...,\n",
       "           [-1.9930, -1.9930, -1.9930,  ..., -1.9926, -1.9927, -1.9926],\n",
       "           [-1.9932, -1.9933, -1.9934,  ..., -1.9925, -1.9927, -1.9926],\n",
       "           [-1.9935, -1.9936, -1.9937,  ..., -1.9926, -1.9928, -1.9926]],\n",
       " \n",
       "          [[-1.9917, -1.9919, -1.9923,  ..., -1.9883, -1.9883, -1.9882],\n",
       "           [-1.9919, -1.9920, -1.9922,  ..., -1.9887, -1.9886, -1.9886],\n",
       "           [-1.9919, -1.9919, -1.9922,  ..., -1.9889, -1.9890, -1.9890],\n",
       "           ...,\n",
       "           [-1.9938, -1.9936, -1.9935,  ..., -1.9862, -1.9854, -1.9848],\n",
       "           [-1.9945, -1.9942, -1.9941,  ..., -1.9865, -1.9857, -1.9850],\n",
       "           [-1.9946, -1.9944, -1.9942,  ..., -1.9869, -1.9861, -1.9852]],\n",
       " \n",
       "          [[-1.9915, -1.9916, -1.9919,  ..., -1.9882, -1.9882, -1.9882],\n",
       "           [-1.9915, -1.9917, -1.9919,  ..., -1.9885, -1.9886, -1.9886],\n",
       "           [-1.9917, -1.9918, -1.9920,  ..., -1.9887, -1.9888, -1.9888],\n",
       "           ...,\n",
       "           [-1.9875, -1.9879, -1.9885,  ..., -1.9843, -1.9843, -1.9843],\n",
       "           [-1.9869, -1.9877, -1.9887,  ..., -1.9843, -1.9843, -1.9843],\n",
       "           [-1.9863, -1.9875, -1.9890,  ..., -1.9843, -1.9843, -1.9843]],\n",
       " \n",
       "          ...,\n",
       " \n",
       "          [[-1.9918, -1.9919, -1.9920,  ..., -1.9887, -1.9887, -1.9887],\n",
       "           [-1.9918, -1.9919, -1.9920,  ..., -1.9886, -1.9886, -1.9886],\n",
       "           [-1.9918, -1.9919, -1.9920,  ..., -1.9886, -1.9886, -1.9886],\n",
       "           ...,\n",
       "           [-1.9960, -1.9960, -1.9960,  ..., -1.9913, -1.9913, -1.9913],\n",
       "           [-1.9960, -1.9960, -1.9960,  ..., -1.9903, -1.9903, -1.9903],\n",
       "           [-1.9960, -1.9960, -1.9960,  ..., -1.9895, -1.9895, -1.9895]],\n",
       " \n",
       "          [[-1.9918, -1.9919, -1.9920,  ..., -1.9886, -1.9886, -1.9886],\n",
       "           [-1.9918, -1.9919, -1.9920,  ..., -1.9885, -1.9885, -1.9885],\n",
       "           [-1.9918, -1.9919, -1.9920,  ..., -1.9885, -1.9885, -1.9885],\n",
       "           ...,\n",
       "           [-1.9960, -1.9960, -1.9960,  ..., -1.9914, -1.9914, -1.9914],\n",
       "           [-1.9960, -1.9960, -1.9960,  ..., -1.9904, -1.9904, -1.9904],\n",
       "           [-1.9960, -1.9960, -1.9960,  ..., -1.9896, -1.9896, -1.9896]],\n",
       " \n",
       "          [[-1.9918, -1.9919, -1.9920,  ..., -1.9886, -1.9886, -1.9886],\n",
       "           [-1.9918, -1.9919, -1.9920,  ..., -1.9885, -1.9885, -1.9885],\n",
       "           [-1.9918, -1.9919, -1.9920,  ..., -1.9885, -1.9885, -1.9885],\n",
       "           ...,\n",
       "           [-1.9947, -1.9937, -1.9925,  ..., -1.9914, -1.9914, -1.9914],\n",
       "           [-1.9947, -1.9937, -1.9925,  ..., -1.9904, -1.9904, -1.9904],\n",
       "           [-1.9947, -1.9937, -1.9925,  ..., -1.9896, -1.9896, -1.9896]]],\n",
       " \n",
       " \n",
       "         [[[-1.9919, -1.9924, -1.9929,  ..., -1.9889, -1.9889, -1.9889],\n",
       "           [-1.9921, -1.9925, -1.9930,  ..., -1.9894, -1.9895, -1.9895],\n",
       "           [-1.9923, -1.9926, -1.9930,  ..., -1.9898, -1.9899, -1.9899],\n",
       "           ...,\n",
       "           [-1.9945, -1.9945, -1.9946,  ..., -1.9937, -1.9939, -1.9938],\n",
       "           [-1.9949, -1.9949, -1.9950,  ..., -1.9934, -1.9936, -1.9935],\n",
       "           [-1.9950, -1.9951, -1.9952,  ..., -1.9931, -1.9933, -1.9931]],\n",
       " \n",
       "          [[-1.9927, -1.9927, -1.9930,  ..., -1.9889, -1.9889, -1.9889],\n",
       "           [-1.9926, -1.9925, -1.9928,  ..., -1.9894, -1.9894, -1.9894],\n",
       "           [-1.9926, -1.9925, -1.9927,  ..., -1.9898, -1.9898, -1.9898],\n",
       "           ...,\n",
       "           [-1.9944, -1.9945, -1.9946,  ..., -1.9870, -1.9860, -1.9850],\n",
       "           [-1.9953, -1.9953, -1.9953,  ..., -1.9872, -1.9863, -1.9854],\n",
       "           [-1.9956, -1.9956, -1.9956,  ..., -1.9875, -1.9869, -1.9863]],\n",
       " \n",
       "          [[-1.9921, -1.9922, -1.9926,  ..., -1.9890, -1.9890, -1.9889],\n",
       "           [-1.9922, -1.9923, -1.9926,  ..., -1.9894, -1.9894, -1.9894],\n",
       "           [-1.9923, -1.9924, -1.9927,  ..., -1.9898, -1.9898, -1.9898],\n",
       "           ...,\n",
       "           [-1.9880, -1.9883, -1.9890,  ..., -1.9850, -1.9850, -1.9850],\n",
       "           [-1.9874, -1.9882, -1.9893,  ..., -1.9850, -1.9850, -1.9850],\n",
       "           [-1.9870, -1.9881, -1.9896,  ..., -1.9850, -1.9850, -1.9850]],\n",
       " \n",
       "          ...,\n",
       " \n",
       "          [[-1.9928, -1.9928, -1.9929,  ..., -1.9891, -1.9891, -1.9891],\n",
       "           [-1.9928, -1.9928, -1.9929,  ..., -1.9893, -1.9893, -1.9893],\n",
       "           [-1.9928, -1.9928, -1.9929,  ..., -1.9895, -1.9895, -1.9895],\n",
       "           ...,\n",
       "           [-1.9969, -1.9969, -1.9969,  ..., -1.9925, -1.9925, -1.9925],\n",
       "           [-1.9969, -1.9969, -1.9969,  ..., -1.9915, -1.9915, -1.9915],\n",
       "           [-1.9969, -1.9969, -1.9969,  ..., -1.9907, -1.9907, -1.9907]],\n",
       " \n",
       "          [[-1.9928, -1.9928, -1.9929,  ..., -1.9891, -1.9891, -1.9891],\n",
       "           [-1.9928, -1.9928, -1.9929,  ..., -1.9892, -1.9892, -1.9892],\n",
       "           [-1.9928, -1.9928, -1.9929,  ..., -1.9894, -1.9894, -1.9894],\n",
       "           ...,\n",
       "           [-1.9970, -1.9970, -1.9970,  ..., -1.9925, -1.9925, -1.9925],\n",
       "           [-1.9970, -1.9970, -1.9970,  ..., -1.9915, -1.9915, -1.9915],\n",
       "           [-1.9970, -1.9970, -1.9970,  ..., -1.9907, -1.9907, -1.9907]],\n",
       " \n",
       "          [[-1.9928, -1.9928, -1.9929,  ..., -1.9890, -1.9890, -1.9890],\n",
       "           [-1.9928, -1.9928, -1.9929,  ..., -1.9891, -1.9891, -1.9891],\n",
       "           [-1.9928, -1.9928, -1.9929,  ..., -1.9893, -1.9893, -1.9893],\n",
       "           ...,\n",
       "           [-1.9953, -1.9943, -1.9931,  ..., -1.9925, -1.9925, -1.9925],\n",
       "           [-1.9953, -1.9943, -1.9931,  ..., -1.9915, -1.9915, -1.9915],\n",
       "           [-1.9953, -1.9943, -1.9931,  ..., -1.9907, -1.9907, -1.9907]]],\n",
       " \n",
       " \n",
       "         [[[-1.9930, -1.9934, -1.9940,  ..., -1.9897, -1.9898, -1.9900],\n",
       "           [-1.9932, -1.9936, -1.9940,  ..., -1.9903, -1.9905, -1.9905],\n",
       "           [-1.9934, -1.9937, -1.9940,  ..., -1.9909, -1.9910, -1.9911],\n",
       "           ...,\n",
       "           [-1.9954, -1.9954, -1.9954,  ..., -1.9948, -1.9949, -1.9948],\n",
       "           [-1.9957, -1.9958, -1.9959,  ..., -1.9946, -1.9948, -1.9947],\n",
       "           [-1.9960, -1.9961, -1.9962,  ..., -1.9944, -1.9946, -1.9945]],\n",
       " \n",
       "          [[-1.9930, -1.9932, -1.9936,  ..., -1.9898, -1.9898, -1.9899],\n",
       "           [-1.9932, -1.9933, -1.9936,  ..., -1.9904, -1.9905, -1.9905],\n",
       "           [-1.9932, -1.9933, -1.9935,  ..., -1.9910, -1.9911, -1.9911],\n",
       "           ...,\n",
       "           [-1.9952, -1.9953, -1.9954,  ..., -1.9881, -1.9870, -1.9860],\n",
       "           [-1.9961, -1.9961, -1.9961,  ..., -1.9882, -1.9873, -1.9864],\n",
       "           [-1.9965, -1.9965, -1.9964,  ..., -1.9888, -1.9881, -1.9873]],\n",
       " \n",
       "          [[-1.9931, -1.9933, -1.9936,  ..., -1.9898, -1.9898, -1.9897],\n",
       "           [-1.9932, -1.9933, -1.9936,  ..., -1.9904, -1.9904, -1.9904],\n",
       "           [-1.9933, -1.9935, -1.9937,  ..., -1.9909, -1.9910, -1.9910],\n",
       "           ...,\n",
       "           [-1.9885, -1.9888, -1.9895,  ..., -1.9861, -1.9861, -1.9861],\n",
       "           [-1.9880, -1.9888, -1.9899,  ..., -1.9861, -1.9861, -1.9861],\n",
       "           [-1.9876, -1.9888, -1.9902,  ..., -1.9861, -1.9861, -1.9861]],\n",
       " \n",
       "          ...,\n",
       " \n",
       "          [[-1.9936, -1.9937, -1.9938,  ..., -1.9910, -1.9910, -1.9910],\n",
       "           [-1.9936, -1.9937, -1.9938,  ..., -1.9911, -1.9911, -1.9911],\n",
       "           [-1.9936, -1.9937, -1.9938,  ..., -1.9912, -1.9912, -1.9912],\n",
       "           ...,\n",
       "           [-1.9975, -1.9975, -1.9975,  ..., -1.9936, -1.9936, -1.9936],\n",
       "           [-1.9975, -1.9975, -1.9975,  ..., -1.9926, -1.9926, -1.9926],\n",
       "           [-1.9975, -1.9975, -1.9975,  ..., -1.9918, -1.9918, -1.9918]],\n",
       " \n",
       "          [[-1.9936, -1.9937, -1.9938,  ..., -1.9910, -1.9910, -1.9910],\n",
       "           [-1.9936, -1.9937, -1.9938,  ..., -1.9911, -1.9911, -1.9911],\n",
       "           [-1.9936, -1.9937, -1.9938,  ..., -1.9912, -1.9912, -1.9912],\n",
       "           ...,\n",
       "           [-1.9976, -1.9976, -1.9976,  ..., -1.9935, -1.9935, -1.9935],\n",
       "           [-1.9976, -1.9976, -1.9976,  ..., -1.9925, -1.9925, -1.9925],\n",
       "           [-1.9976, -1.9976, -1.9976,  ..., -1.9917, -1.9917, -1.9917]],\n",
       " \n",
       "          [[-1.9936, -1.9937, -1.9938,  ..., -1.9908, -1.9908, -1.9908],\n",
       "           [-1.9936, -1.9937, -1.9938,  ..., -1.9909, -1.9909, -1.9909],\n",
       "           [-1.9936, -1.9937, -1.9938,  ..., -1.9910, -1.9910, -1.9910],\n",
       "           ...,\n",
       "           [-1.9956, -1.9947, -1.9934,  ..., -1.9935, -1.9935, -1.9935],\n",
       "           [-1.9956, -1.9947, -1.9934,  ..., -1.9925, -1.9925, -1.9925],\n",
       "           [-1.9956, -1.9947, -1.9934,  ..., -1.9917, -1.9917, -1.9917]]]]),\n",
       " tensor([[[[-1.9914, -1.9918, -1.9923,  ..., -1.9882, -1.9883, -1.9884],\n",
       "           [-1.9917, -1.9920, -1.9924,  ..., -1.9884, -1.9885, -1.9886],\n",
       "           [-1.9919, -1.9921, -1.9924,  ..., -1.9886, -1.9888, -1.9889],\n",
       "           ...,\n",
       "           [-1.9930, -1.9930, -1.9930,  ..., -1.9926, -1.9927, -1.9926],\n",
       "           [-1.9932, -1.9933, -1.9934,  ..., -1.9925, -1.9927, -1.9926],\n",
       "           [-1.9935, -1.9936, -1.9937,  ..., -1.9926, -1.9928, -1.9926]],\n",
       " \n",
       "          [[-1.9914, -1.9918, -1.9923,  ..., -1.9882, -1.9883, -1.9884],\n",
       "           [-1.9917, -1.9920, -1.9924,  ..., -1.9884, -1.9885, -1.9886],\n",
       "           [-1.9919, -1.9921, -1.9924,  ..., -1.9886, -1.9888, -1.9889],\n",
       "           ...,\n",
       "           [-1.9930, -1.9930, -1.9930,  ..., -1.9926, -1.9927, -1.9926],\n",
       "           [-1.9932, -1.9933, -1.9934,  ..., -1.9925, -1.9927, -1.9926],\n",
       "           [-1.9935, -1.9936, -1.9937,  ..., -1.9926, -1.9928, -1.9926]],\n",
       " \n",
       "          [[-1.9917, -1.9919, -1.9923,  ..., -1.9884, -1.9885, -1.9886],\n",
       "           [-1.9919, -1.9920, -1.9924,  ..., -1.9885, -1.9886, -1.9888],\n",
       "           [-1.9919, -1.9920, -1.9923,  ..., -1.9886, -1.9889, -1.9891],\n",
       "           ...,\n",
       "           [-1.9937, -1.9936, -1.9935,  ..., -1.9847, -1.9847, -1.9846],\n",
       "           [-1.9940, -1.9939, -1.9939,  ..., -1.9845, -1.9845, -1.9845],\n",
       "           [-1.9939, -1.9940, -1.9941,  ..., -1.9847, -1.9845, -1.9845]],\n",
       " \n",
       "          ...,\n",
       " \n",
       "          [[-1.9919, -1.9920, -1.9921,  ..., -1.9886, -1.9886, -1.9886],\n",
       "           [-1.9919, -1.9920, -1.9921,  ..., -1.9885, -1.9885, -1.9885],\n",
       "           [-1.9919, -1.9920, -1.9921,  ..., -1.9885, -1.9885, -1.9885],\n",
       "           ...,\n",
       "           [-1.9942, -1.9937, -1.9931,  ..., -1.9914, -1.9914, -1.9914],\n",
       "           [-1.9942, -1.9937, -1.9931,  ..., -1.9904, -1.9904, -1.9904],\n",
       "           [-1.9942, -1.9937, -1.9931,  ..., -1.9896, -1.9896, -1.9896]],\n",
       " \n",
       "          [[-1.9918, -1.9919, -1.9920,  ..., -1.9886, -1.9886, -1.9886],\n",
       "           [-1.9918, -1.9919, -1.9920,  ..., -1.9885, -1.9885, -1.9885],\n",
       "           [-1.9918, -1.9919, -1.9920,  ..., -1.9885, -1.9885, -1.9885],\n",
       "           ...,\n",
       "           [-1.9947, -1.9937, -1.9925,  ..., -1.9914, -1.9914, -1.9914],\n",
       "           [-1.9947, -1.9937, -1.9925,  ..., -1.9904, -1.9904, -1.9904],\n",
       "           [-1.9947, -1.9937, -1.9925,  ..., -1.9896, -1.9896, -1.9896]],\n",
       " \n",
       "          [[-1.9918, -1.9919, -1.9920,  ..., -1.9886, -1.9886, -1.9886],\n",
       "           [-1.9918, -1.9919, -1.9920,  ..., -1.9885, -1.9885, -1.9885],\n",
       "           [-1.9918, -1.9919, -1.9920,  ..., -1.9885, -1.9885, -1.9885],\n",
       "           ...,\n",
       "           [-1.9947, -1.9937, -1.9925,  ..., -1.9914, -1.9914, -1.9914],\n",
       "           [-1.9947, -1.9937, -1.9925,  ..., -1.9904, -1.9904, -1.9904],\n",
       "           [-1.9947, -1.9937, -1.9925,  ..., -1.9896, -1.9896, -1.9896]]],\n",
       " \n",
       " \n",
       "         [[[-1.9919, -1.9924, -1.9929,  ..., -1.9889, -1.9889, -1.9889],\n",
       "           [-1.9921, -1.9925, -1.9930,  ..., -1.9894, -1.9895, -1.9895],\n",
       "           [-1.9923, -1.9926, -1.9930,  ..., -1.9898, -1.9899, -1.9899],\n",
       "           ...,\n",
       "           [-1.9945, -1.9945, -1.9946,  ..., -1.9937, -1.9939, -1.9938],\n",
       "           [-1.9949, -1.9949, -1.9950,  ..., -1.9934, -1.9936, -1.9935],\n",
       "           [-1.9950, -1.9951, -1.9952,  ..., -1.9931, -1.9933, -1.9931]],\n",
       " \n",
       "          [[-1.9919, -1.9924, -1.9929,  ..., -1.9889, -1.9889, -1.9889],\n",
       "           [-1.9921, -1.9925, -1.9930,  ..., -1.9894, -1.9895, -1.9895],\n",
       "           [-1.9923, -1.9926, -1.9930,  ..., -1.9898, -1.9899, -1.9899],\n",
       "           ...,\n",
       "           [-1.9945, -1.9945, -1.9946,  ..., -1.9937, -1.9939, -1.9938],\n",
       "           [-1.9949, -1.9949, -1.9950,  ..., -1.9934, -1.9936, -1.9935],\n",
       "           [-1.9950, -1.9951, -1.9952,  ..., -1.9931, -1.9933, -1.9931]],\n",
       " \n",
       "          [[-1.9922, -1.9925, -1.9929,  ..., -1.9888, -1.9887, -1.9888],\n",
       "           [-1.9924, -1.9926, -1.9930,  ..., -1.9893, -1.9894, -1.9894],\n",
       "           [-1.9925, -1.9925, -1.9929,  ..., -1.9896, -1.9898, -1.9898],\n",
       "           ...,\n",
       "           [-1.9945, -1.9945, -1.9946,  ..., -1.9857, -1.9859, -1.9860],\n",
       "           [-1.9948, -1.9949, -1.9951,  ..., -1.9854, -1.9856, -1.9857],\n",
       "           [-1.9951, -1.9951, -1.9953,  ..., -1.9854, -1.9855, -1.9854]],\n",
       " \n",
       "          ...,\n",
       " \n",
       "          [[-1.9928, -1.9929, -1.9930,  ..., -1.9890, -1.9890, -1.9890],\n",
       "           [-1.9928, -1.9929, -1.9930,  ..., -1.9891, -1.9891, -1.9891],\n",
       "           [-1.9928, -1.9929, -1.9930,  ..., -1.9893, -1.9893, -1.9893],\n",
       "           ...,\n",
       "           [-1.9952, -1.9947, -1.9941,  ..., -1.9925, -1.9925, -1.9925],\n",
       "           [-1.9952, -1.9947, -1.9941,  ..., -1.9915, -1.9915, -1.9915],\n",
       "           [-1.9952, -1.9947, -1.9941,  ..., -1.9907, -1.9907, -1.9907]],\n",
       " \n",
       "          [[-1.9928, -1.9928, -1.9929,  ..., -1.9890, -1.9890, -1.9890],\n",
       "           [-1.9928, -1.9928, -1.9929,  ..., -1.9891, -1.9891, -1.9891],\n",
       "           [-1.9928, -1.9928, -1.9929,  ..., -1.9893, -1.9893, -1.9893],\n",
       "           ...,\n",
       "           [-1.9953, -1.9943, -1.9931,  ..., -1.9925, -1.9925, -1.9925],\n",
       "           [-1.9953, -1.9943, -1.9931,  ..., -1.9915, -1.9915, -1.9915],\n",
       "           [-1.9953, -1.9943, -1.9931,  ..., -1.9907, -1.9907, -1.9907]],\n",
       " \n",
       "          [[-1.9928, -1.9928, -1.9929,  ..., -1.9890, -1.9890, -1.9890],\n",
       "           [-1.9928, -1.9928, -1.9929,  ..., -1.9891, -1.9891, -1.9891],\n",
       "           [-1.9928, -1.9928, -1.9929,  ..., -1.9893, -1.9893, -1.9893],\n",
       "           ...,\n",
       "           [-1.9953, -1.9943, -1.9931,  ..., -1.9925, -1.9925, -1.9925],\n",
       "           [-1.9953, -1.9943, -1.9931,  ..., -1.9915, -1.9915, -1.9915],\n",
       "           [-1.9953, -1.9943, -1.9931,  ..., -1.9907, -1.9907, -1.9907]]],\n",
       " \n",
       " \n",
       "         [[[-1.9930, -1.9934, -1.9940,  ..., -1.9897, -1.9898, -1.9900],\n",
       "           [-1.9932, -1.9936, -1.9940,  ..., -1.9903, -1.9905, -1.9905],\n",
       "           [-1.9934, -1.9937, -1.9940,  ..., -1.9909, -1.9910, -1.9911],\n",
       "           ...,\n",
       "           [-1.9954, -1.9954, -1.9954,  ..., -1.9948, -1.9949, -1.9948],\n",
       "           [-1.9957, -1.9958, -1.9959,  ..., -1.9946, -1.9948, -1.9947],\n",
       "           [-1.9960, -1.9961, -1.9962,  ..., -1.9944, -1.9946, -1.9945]],\n",
       " \n",
       "          [[-1.9930, -1.9934, -1.9940,  ..., -1.9897, -1.9898, -1.9900],\n",
       "           [-1.9932, -1.9936, -1.9940,  ..., -1.9903, -1.9905, -1.9905],\n",
       "           [-1.9934, -1.9937, -1.9940,  ..., -1.9909, -1.9910, -1.9911],\n",
       "           ...,\n",
       "           [-1.9954, -1.9954, -1.9954,  ..., -1.9948, -1.9949, -1.9948],\n",
       "           [-1.9957, -1.9958, -1.9959,  ..., -1.9946, -1.9948, -1.9947],\n",
       "           [-1.9960, -1.9961, -1.9962,  ..., -1.9944, -1.9946, -1.9945]],\n",
       " \n",
       "          [[-1.9931, -1.9934, -1.9938,  ..., -1.9900, -1.9900, -1.9901],\n",
       "           [-1.9933, -1.9935, -1.9938,  ..., -1.9905, -1.9905, -1.9906],\n",
       "           [-1.9934, -1.9934, -1.9938,  ..., -1.9909, -1.9910, -1.9912],\n",
       "           ...,\n",
       "           [-1.9957, -1.9956, -1.9956,  ..., -1.9871, -1.9873, -1.9874],\n",
       "           [-1.9960, -1.9960, -1.9961,  ..., -1.9868, -1.9869, -1.9870],\n",
       "           [-1.9961, -1.9962, -1.9963,  ..., -1.9866, -1.9866, -1.9866]],\n",
       " \n",
       "          ...,\n",
       " \n",
       "          [[-1.9937, -1.9938, -1.9939,  ..., -1.9908, -1.9908, -1.9908],\n",
       "           [-1.9937, -1.9938, -1.9939,  ..., -1.9909, -1.9909, -1.9909],\n",
       "           [-1.9937, -1.9938, -1.9939,  ..., -1.9910, -1.9910, -1.9910],\n",
       "           ...,\n",
       "           [-1.9954, -1.9949, -1.9943,  ..., -1.9935, -1.9935, -1.9935],\n",
       "           [-1.9954, -1.9949, -1.9943,  ..., -1.9925, -1.9925, -1.9925],\n",
       "           [-1.9954, -1.9949, -1.9943,  ..., -1.9917, -1.9917, -1.9917]],\n",
       " \n",
       "          [[-1.9936, -1.9937, -1.9938,  ..., -1.9908, -1.9908, -1.9908],\n",
       "           [-1.9936, -1.9937, -1.9938,  ..., -1.9909, -1.9909, -1.9909],\n",
       "           [-1.9936, -1.9937, -1.9938,  ..., -1.9910, -1.9910, -1.9910],\n",
       "           ...,\n",
       "           [-1.9956, -1.9947, -1.9934,  ..., -1.9935, -1.9935, -1.9935],\n",
       "           [-1.9956, -1.9947, -1.9934,  ..., -1.9925, -1.9925, -1.9925],\n",
       "           [-1.9956, -1.9947, -1.9934,  ..., -1.9917, -1.9917, -1.9917]],\n",
       " \n",
       "          [[-1.9936, -1.9937, -1.9938,  ..., -1.9908, -1.9908, -1.9908],\n",
       "           [-1.9936, -1.9937, -1.9938,  ..., -1.9909, -1.9909, -1.9909],\n",
       "           [-1.9936, -1.9937, -1.9938,  ..., -1.9910, -1.9910, -1.9910],\n",
       "           ...,\n",
       "           [-1.9956, -1.9947, -1.9934,  ..., -1.9935, -1.9935, -1.9935],\n",
       "           [-1.9956, -1.9947, -1.9934,  ..., -1.9925, -1.9925, -1.9925],\n",
       "           [-1.9956, -1.9947, -1.9934,  ..., -1.9917, -1.9917, -1.9917]]]])]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.__getitem__(\"tumblr_nkaeprvFbf1u680rpo1_400.gif\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndentationError",
     "evalue": "unindent does not match any outer indentation level (<tokenize>, line 48)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<tokenize>\"\u001b[0;36m, line \u001b[0;32m48\u001b[0m\n\u001b[0;31m    test_data_path = os.path.join(self.dataframe_dir, 'Test_action_question.csv')\u001b[0m\n\u001b[0m    ^\u001b[0m\n\u001b[0;31mIndentationError\u001b[0m\u001b[0;31m:\u001b[0m unindent does not match any outer indentation level\n"
     ]
    }
   ],
   "source": [
    "class TGIFDataset(Dataset):\n",
    "    def __init__(self, dataset_name='train', data_type=None, dataframe_dir=None, vocab_dir=None):\n",
    "        self.dataframe_dir = dataframe_dir # of the form data/tgif/vocabulary\n",
    "        self.vocab_dir = vocab_dir # of the form data/tgif/dataframe\n",
    "        self.data_type = data_type # 'TRANS'\n",
    "        self.dataset_name = dataset_name # 'train' or 'val' or 'test'\n",
    "\n",
    "        self.csv = self.read_from_csvfile()\n",
    "        self.header2idx = self.header2idx()\n",
    "        self.gif_names = self.csv[:,self.header2idx['gif_name']]\n",
    "        self.gif_tensor = None\n",
    "        self.questions = self.csv[:,self.header2idx['question']]\n",
    "        self.answers = self.csv[:,self.header2idx['answer']]\n",
    "        self.mc_options = self.csv[:,self.header2idx['a1']:header2idx['a5']+1]\n",
    "        ## GIF LOADER ##\n",
    "        ## NOTE: May have to change the relative path of gif dir as \n",
    "        ## an extra argument to TGIF class init\n",
    "        loader  = TGIF(cfg, \"train\")\n",
    "        self.get_gif_tensor = loader.__getitem__\n",
    "        \n",
    "    def __getitem__(self, i): # whats the argument for this\n",
    "    \tgif_path = os.path.join(self.dataframe_dir, 'gif_tensors')\n",
    "    \t#pick up ith gif_tensor\n",
    "        #NOTE: gif_path is only the gif name, not the relative path\n",
    "        # REturn value: tuple (slow frames, fast frames) where frame -> (t, 3, h, w)\n",
    "        gif_tensor = self.get_gif_tensor(gif_path)\n",
    "    \treturn self.gif_tensor, self.questions[i], self.mc_options[i], self.answers[i]\n",
    "\n",
    "    def header2idx(self):\n",
    "    \treturn {'gif_name':0,'question':1,'a1':2,'a2':3,'a3':4,'a4':5,'a5':6,'answer':7,'vid_id':8,'key':9}\n",
    "\n",
    "    def read_from_csvfile(self):\n",
    "        assert self.data_type in ['TRANS', 'ACTION'] # ACTION just for starting, will be using TRANS finally\n",
    "\n",
    "        self.total_q=[]\n",
    "        if self.data_type=='TRANS':\n",
    "            train_data_path = os.path.join(self.dataframe_dir, 'Train_transition_question.csv')\n",
    "            test_data_path = os.path.join(self.dataframe_dir, 'Test_transition_question.csv')\n",
    "\n",
    "            \n",
    "            with open(os.path.join(self.dataframe_dir, 'Total_transition_question.csv')) as file:\n",
    "            \tcsv_reader = csv.reader(file, delimiter='\\t')\n",
    "            \tfor row in csv_reader:\n",
    "            \t\tself.total_q.append(row)\n",
    "\n",
    "        elif self.data_type=='ACTION':\n",
    "         \ttrain_data_path = os.path.join(self.dataframe_dir, 'Train_action_question.csv')\n",
    "            test_data_path = os.path.join(self.dataframe_dir, 'Test_action_question.csv')\n",
    "\n",
    "            with open(os.path.join(self.dataframe_dir, 'Total_action_question.csv')) as file:\n",
    "            \tcsv_reader = csv.reader(file, delimiter='\\t')\n",
    "            \tfor row in csv_reader:\n",
    "            \t\tself.total_q.append(row)\n",
    "        \n",
    "        self.total_q.pop(0)\n",
    "\n",
    "        assert_exists(train_data_path)\n",
    "        assert_exits(test_data_path)\n",
    "\n",
    "        csv_data=[]\n",
    "        if self.dataset_name=='train':\n",
    "        \twith open(train_data_path) as file:\n",
    "        \t\tcsv_reader = csv.reader(file, delimiter='\\t')\n",
    "        \t\tfor row in csv_reader:\n",
    "        \t\t\tcsv_data.append(row)\n",
    "        elif self.dataset_name=='test':\n",
    "        \twith open(test_data_path) as file:\n",
    "        \t\tcsv_reader = csv.reader(file, delimiter='\\t')\n",
    "        \t\tfor row in csv_reader:\n",
    "        \t\t\tcsv_data.append(row)\n",
    "        csv_data.pop(0)\n",
    "\n",
    "        return np.asarray(csv_data)\n",
    "    '''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import csv\n",
    "import sys\n",
    "import numpy as np\n",
    "sys.path.insert(1, '../')\n",
    "from lxrt.SlowFast.slowfast.config.defaults import get_cfg\n",
    "from lxrt.SlowFast.slowfast.datasets.tgif_direct import TGIF\n",
    "\n",
    "\n",
    "cfg = get_cfg()\n",
    "cfg_file = \"../lxrt/SlowFast/configs/Kinetics/c2/SLOWFAST_8x8_R50.yaml\"\n",
    "cfg.merge_from_file(cfg_file)\n",
    "\n",
    "class FrameQADataset(object):\n",
    "    def __init__(self, dataset_name='train', data_type=None, dataframe_dir=None, vocab_dir=None, category =\"frameqa\" ):\n",
    "        self.dataframe_dir = dataframe_dir # of the form data/tgif/vocabulary\n",
    "        self.vocab_dir = vocab_dir # of the form data/tgif/dataframe\n",
    "        self.data_type = data_type # 'TRANS'\n",
    "        self.dataset_name = dataset_name # 'train' or 'val' or 'test'\n",
    "\n",
    "        self.csv, all_data = self.read_from_csvfile(category)\n",
    "        self.header2idx = self.header2idx()\n",
    "        self.gif_names = self.csv[:,self.header2idx['gif_name']]\n",
    "        self.gif_tensor = None\n",
    "        self.questions = self.csv[:,self.header2idx['question']]\n",
    "        self.answer = self.csv[:,self.header2idx['answer']]\n",
    "        self._build_ans_vocab(all_data[:,self.header2idx['answer']])\n",
    "        ## GIF LOADER ##\n",
    "        ## NOTE: May have to change the relative path of gif dir as \n",
    "        ## an extra argument to TGIF class init\n",
    "        root_path = \"/users/cdwivedi/RL_EXP/IDL/project/tgif-qa/code/dataset/tgif/frame_gifs/\"+dataset_name+\"/\"\n",
    "        loader  = TGIF(cfg, \"train\",root_path=root_path )\n",
    "        self.get_gif_tensor = loader.__getitem__\n",
    "        \n",
    "    def _build_ans_vocab(self, all_answers):\n",
    "        vocab = set()\n",
    "        for ans in all_answers:\n",
    "            vocab.add(str(ans))\n",
    "        self.vocab = sorted(list(vocab))\n",
    "        self.id2ans = self.vocab\n",
    "        self.ans2id = dict(zip(self.vocab, np.arange(len(self.vocab))))\n",
    "        self.vocab_len = len(self.vocab)\n",
    "        self.num_answers = self.vocab_len\n",
    "        self.label2ans = self.id2ans\n",
    "        \n",
    "    def __getitem__(self, i): # whats the argument for this\n",
    "        gif_path = self.gif_names[i]\n",
    "        #pick up ith gif_tensor\n",
    "        #NOTE: gif_path is only the gif name, not the relative path\n",
    "        # REturn value: tuple (slow frames, fast frames) where frame -> (t, 3, h, w)\n",
    "        gif_tensor = self.get_gif_tensor(gif_path)\n",
    "        \n",
    "        return gif_tensor, self.questions[i], self.ans2id[self.answer[i]]\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.questions)\n",
    "    \n",
    "    def header2idx(self):\n",
    "        return {'gif_name':0,'question':1,'answer':2}\n",
    "\n",
    "    def read_from_csvfile(self, category=None):\n",
    "        print(category)\n",
    "        train_data_path = os.path.join(self.dataframe_dir, 'Train_'+category+'_question.csv')\n",
    "        test_data_path = os.path.join(self.dataframe_dir, 'Test_'+category+'_question.csv')\n",
    "        total_data_path = os.path.join(self.dataframe_dir, 'Total_'+category+'_question.csv')\n",
    "        csv_data=[]\n",
    "        if self.dataset_name=='train':\n",
    "            with open(train_data_path) as file:\n",
    "                csv_reader = csv.reader(file, delimiter='\\t')\n",
    "                for row in csv_reader:\n",
    "                    csv_data.append(row)\n",
    "        elif self.dataset_name=='test':\n",
    "            with open(test_data_path) as file:\n",
    "                csv_reader = csv.reader(file, delimiter='\\t')\n",
    "                for row in csv_reader:\n",
    "                    csv_data.append(row)\n",
    "        csv_data.pop(0)\n",
    "        total_csv_data=[]\n",
    "        with open(total_data_path) as file:\n",
    "            csv_reader = csv.reader(file, delimiter='\\t')\n",
    "            for row in csv_reader:\n",
    "                total_csv_data.append(row)\n",
    "\n",
    "        total_csv_data.pop(0)\n",
    "        return np.asarray(csv_data), np.asarray(total_csv_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path= \"../../../../../../IDL/project/tgif-qa/dataset/\"\n",
    "file = \"Train_frameqa_question.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_q =[]\n",
    "with open(data_path+file) as f:\n",
    "    reader = csv.reader(f, delimiter=\"\\t\")\n",
    "    for row in reader:\n",
    "        total_q.append(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "total_q = np.asarray(total_q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frameqa\n"
     ]
    }
   ],
   "source": [
    "d = FrameQADataset(dataframe_dir=\"../../../../../../IDL/project/tgif-qa/dataset/\", dataset_name=\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1746"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d.vocab_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'tasks'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-6c9aa99fa495>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;31m#from param import args\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;31m#from pretrain.qa_answer_table import load_lxmert_qa\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtasks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvqa_model\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mVQAModel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtasks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvqa_data\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mVQADataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mVQATorchDataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mVQAEvaluator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mFrameQADataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'tasks'"
     ]
    }
   ],
   "source": [
    "\n",
    "import os\n",
    "import collections\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data.dataloader import DataLoader\n",
    "from tqdm import tqdm\n",
    "\n",
    "#from param import args\n",
    "#from pretrain.qa_answer_table import load_lxmert_qa\n",
    "from tasks.vqa_model import VQAModel\n",
    "from tasks.vqa_data import VQADataset, VQATorchDataset, VQAEvaluator, FrameQADataset\n",
    "\n",
    "DataTuple = collections.namedtuple(\"DataTuple\", 'dataset loader evaluator')\n",
    "from logger_utils import logger as log\n",
    "logger = log(\"TEST1\")\n",
    "\n",
    "# def get_data_tuple(splits: str, bs:int, shuffle=False, drop_last=False) -> DataTuple:\n",
    "#     dset = 7(splits)\n",
    "#     tset = VQATorchDataset(dset)\n",
    "#     evaluator = VQAEvaluator(dset)\n",
    "#     data_loader = DataLoader(\n",
    "#         tset, batch_size=bs,\n",
    "#         shuffle=shuffle, num_workers=args.num_workers,\n",
    "#         drop_last=drop_last, pin_memory=True\n",
    "#     )\n",
    "\n",
    "#     return DataTuple(dataset=dset, loader=data_loader, evaluator=evaluator)\n",
    "\n",
    "def get_data_tuple(args_train, bs=32,shuffle=False, drop_last=False) -> DataTuple:\n",
    "    dset = FrameQADataset(dataframe_dir=\"../../tgif-qa/dataset/\", \\\n",
    "                          dataset_name=\"test\")\n",
    "    data_loader = DataLoader(\n",
    "        dset, batch_size=bs,\n",
    "        shuffle=shuffle, num_workers=args.num_workers,\n",
    "        drop_last=drop_last, pin_memory=True\n",
    "    )\n",
    "    return DataTuple(dataset=dset, loader=data_loader, evaluator=None)\n",
    "\n",
    "\n",
    "class VQA:\n",
    "    def __init__(self):\n",
    "        # Datasets\n",
    "        print(\"Fetching data\")\n",
    "        self.train_tuple = get_data_tuple(\n",
    "            args.train, bs=args.batch_size, shuffle=True, drop_last=True\n",
    "        )\n",
    "        print(\"Got data\")\n",
    "        if args.valid != \"\":\n",
    "            self.valid_tuple = get_data_tuple(\n",
    "                args.valid, bs=8,\n",
    "                shuffle=False, drop_last=False\n",
    "            )\n",
    "        else:\n",
    "            self.valid_tuple = None\n",
    "        print(\"Got data\")\n",
    "        \n",
    "        # Model\n",
    "        print(\"Making model\")\n",
    "        self.model = VQAModel(self.train_tuple.dataset.num_answers)\n",
    "        print(\"Ready model\")\n",
    "        # Print model info:\n",
    "        print(\"Num of answers:\")\n",
    "        print(self.train_tuple.dataset.num_answers)\n",
    "        # print(\"Model info:\")\n",
    "        # print(self.model)\n",
    "\n",
    "        # Load pre-trained weights\n",
    "        if args.load_lxmert is not None:\n",
    "            self.model.lxrt_encoder.load(args.load_lxmert)\n",
    "        if args.load_lxmert_qa is not None:\n",
    "            load_lxmert_qa(args.load_lxmert_qa, self.model,\n",
    "                           label2ans=self.train_tuple.dataset.label2ans)\n",
    "        \n",
    "        # GPU options\n",
    "        self.model = self.model.cuda()\n",
    "        if args.multiGPU:\n",
    "            self.model.lxrt_encoder.multi_gpu()\n",
    "\n",
    "        # Loss and Optimizer\n",
    "        self.bce_loss = nn.BCEWithLogitsLoss()\n",
    "        if 'bert' in args.optim:\n",
    "            batch_per_epoch = len(self.train_tuple.loader)\n",
    "            t_total = int(batch_per_epoch * args.epochs)\n",
    "            print(\"BertAdam Total Iters: %d\" % t_total)\n",
    "            from lxrt.optimization import BertAdam\n",
    "            self.optim = BertAdam(list(self.model.parameters()),\n",
    "                                  lr=args.lr,\n",
    "                                  warmup=0.1,\n",
    "                                  t_total=t_total)\n",
    "        else:\n",
    "            self.optim = args.optimizer(self.model.parameters(), args.lr)\n",
    "        \n",
    "        # Output Directory\n",
    "        self.output = args.output\n",
    "        os.makedirs(self.output, exist_ok=True)\n",
    "\n",
    "    def train(self, train_tuple, eval_tuple):\n",
    "        dset, loader, evaluator = train_tuple\n",
    "        iter_wrapper = (lambda x: tqdm(x, total=len(loader))) if args.tqdm else (lambda x: x)\n",
    "\n",
    "        best_valid = 0.\n",
    "        flag = True\n",
    "        for epoch in range(args.epochs):\n",
    "            quesid2ans = {}\n",
    "            correct = 0\n",
    "            total_loss = 0\n",
    "            total = 0\n",
    "            print(\"Len of the dataloader: \", len(loader))\n",
    "#             Our new TGIFQA-Dataset returns:\n",
    "#             return gif_tensor, self.questions[i], self.ans2id[self.answer[i]]\n",
    "            for i, (feats1, feats2, sent, target) in iter_wrapper(enumerate(loader)):\n",
    "                ques_id, boxes = -1, None\n",
    "                self.model.train()\n",
    "                self.optim.zero_grad()\n",
    "\n",
    "                \n",
    "                feats1, feats2, target = feats1.cuda(), feats2.cuda(), target.cuda()\n",
    "                feats = [feats1, feats2]\n",
    "                \n",
    "                logit = self.model(feats, boxes, sent)\n",
    "                assert logit.dim() == target.dim() == 2\n",
    "                loss = self.bce_loss(logit, target)\n",
    "                loss = loss * logit.size(1)\n",
    "        \n",
    "                total_loss += loss.item()\n",
    "                \n",
    "                loss.backward()\n",
    "                nn.utils.clip_grad_norm_(self.model.parameters(), 5.)\n",
    "                self.optim.step()\n",
    "\n",
    "                score, label = logit.max(1)\n",
    "                score_t, target = target.max(1)\n",
    "                correct += (label == target).sum().cpu().numpy()\n",
    "                total += len(label)\n",
    "                if epoch > 4:\n",
    "                    for l,s,t in zip(label, sent, target):\n",
    "                        print(l)\n",
    "                        print(s)\n",
    "                        print(\"Prediction\", loader.dataset.label2ans[int(l.cpu().numpy())])\n",
    "                        print(\"Answer\", loader.dataset.label2ans[int(t.cpu().numpy())])\n",
    "            logger.log(total_loss/len(loader), correct/len(loader)*100, epoch)\n",
    "            print(\"==\"*30)\n",
    "            print(\"Accuracy = \" , correct/total*100)\n",
    "            print(\"Loss =\" , total_loss/total)\n",
    "            print(\"==\"*30)\n",
    "#             log_str = \"\\nEpoch %d: Train %0.2f\\n\" % (epoch, evaluator.evaluate(quesid2ans) * 100.)\n",
    "\n",
    "#             if self.valid_tuple is not None:  # Do Validation\n",
    "#                 valid_score = self.evaluate(eval_tuple)\n",
    "#                 if valid_score > best_valid:\n",
    "#                     best_valid = valid_score\n",
    "#                     self.save(\"BEST\")\n",
    "\n",
    "#                 log_str += \"Epoch %d: Valid %0.2f\\n\" % (epoch, valid_score * 100.) + \\\n",
    "#                            \"Epoch %d: Best %0.2f\\n\" % (epoch, best_valid * 100.)\n",
    "\n",
    "#             print(log_str, end='')\n",
    "\n",
    "#             with open(self.output + \"/log.log\", 'a') as f:\n",
    "#                 f.write(log_str)\n",
    "#                 f.flush()\n",
    "\n",
    "            self.save(str(epoch))\n",
    "\n",
    "    def predict(self, eval_tuple: DataTuple, dump=None):\n",
    "        \"\"\"\n",
    "        Predict the answers to questions in a data split.\n",
    "\n",
    "        :param eval_tuple: The data tuple to be evaluated.\n",
    "        :param dump: The path of saved file to dump results.\n",
    "        :return: A dict of question_id to answer.\n",
    "        \"\"\"\n",
    "        self.model.eval()\n",
    "        dset, loader, evaluator = eval_tuple\n",
    "        quesid2ans = {}\n",
    "        for i, datum_tuple in enumerate(loader):\n",
    "            ques_id, feats, boxes, sent = datum_tuple[:4]   # Avoid seeing ground truth\n",
    "            with torch.no_grad():\n",
    "                feats, boxes = feats.cuda(), boxes.cuda()\n",
    "                logit = self.model(feats, boxes, sent)\n",
    "                score, label = logit.max(1)\n",
    "                for qid, l in zip(ques_id, label.cpu().numpy()):\n",
    "                    ans = dset.label2ans[l]\n",
    "                    quesid2ans[qid.item()] = ans\n",
    "        if dump is not None:\n",
    "            evaluator.dump_result(quesid2ans, dump)\n",
    "        return quesid2ans\n",
    "\n",
    "    def evaluate(self, eval_tuple: DataTuple, dump=None):\n",
    "        \"\"\"Evaluate all data in data_tuple.\"\"\"\n",
    "        quesid2ans = self.predict(eval_tuple, dump)\n",
    "        return eval_tuple.evaluator.evaluate(quesid2ans)\n",
    "\n",
    "    @staticmethod\n",
    "    def oracle_score(data_tuple):\n",
    "        dset, loader, evaluator = data_tuple\n",
    "        quesid2ans = {}\n",
    "        for i, (ques_id, feats, boxes, sent, target) in enumerate(loader):\n",
    "            _, label = target.max(1)\n",
    "            for qid, l in zip(ques_id, label.cpu().numpy()):\n",
    "                ans = dset.label2ans[l]\n",
    "                quesid2ans[qid.item()] = ans\n",
    "        return evaluator.evaluate(quesid2ans)\n",
    "\n",
    "    def save(self, name):\n",
    "        torch.save(self.model.state_dict(),\n",
    "                   os.path.join(self.output, \"%s.pth\" % name))\n",
    "\n",
    "    def load(self, path):\n",
    "        print(\"Load model from %s\" % path)\n",
    "        state_dict = torch.load(\"%s.pth\" % path)\n",
    "        self.model.load_state_dict(state_dict)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "# insert at 1, 0 is the script path (or '' in REPL)\n",
    "sys.path.insert(1, '../')\n",
    "from lxrt.SlowFast.slowfast.config.defaults import get_cfg\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "usage: ipykernel_launcher.py [-h] [--train TRAIN] [--valid VALID]\n",
      "                             [--test TEST] [--batchSize BATCH_SIZE]\n",
      "                             [--optim OPTIM] [--lr LR] [--epochs EPOCHS]\n",
      "                             [--dropout DROPOUT] [--seed SEED]\n",
      "                             [--output OUTPUT] [--fast] [--tiny] [--tqdm]\n",
      "                             [--load LOAD] [--loadLXMERT LOAD_LXMERT]\n",
      "                             [--loadLXMERTQA LOAD_LXMERT_QA] [--fromScratch]\n",
      "                             [--mceLoss] [--llayers LLAYERS]\n",
      "                             [--xlayers XLAYERS] [--rlayers RLAYERS]\n",
      "                             [--taskMatched] [--taskMaskLM] [--taskObjPredict]\n",
      "                             [--taskQA] [--visualLosses VISUAL_LOSSES]\n",
      "                             [--qaSets QA_SETS]\n",
      "                             [--wordMaskRate WORD_MASK_RATE]\n",
      "                             [--objMaskRate OBJ_MASK_RATE] [--multiGPU]\n",
      "                             [--numWorkers NUM_WORKERS]\n",
      "ipykernel_launcher.py: error: unrecognized arguments: -f /users/cdwivedi/.local/share/jupyter/runtime/kernel-edcc6253-8fcd-4ad9-8951-5d1723147af0.json\n"
     ]
    },
    {
     "ename": "SystemExit",
     "evalue": "2",
     "output_type": "error",
     "traceback": [
      "An exception has occurred, use %tb to see the full traceback.\n",
      "\u001b[0;31mSystemExit\u001b[0m\u001b[0;31m:\u001b[0m 2\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from tasks.vqa_model import VQAModel\n",
    "\n",
    "\n",
    "#VQAModel(1704)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "usage: ipykernel_launcher.py [-h] [--train TRAIN] [--valid VALID]\n",
      "                             [--test TEST] [--batchSize BATCH_SIZE]\n",
      "                             [--optim OPTIM] [--lr LR] [--epochs EPOCHS]\n",
      "                             [--dropout DROPOUT] [--seed SEED]\n",
      "                             [--output OUTPUT] [--fast] [--tiny] [--tqdm]\n",
      "                             [--load LOAD] [--loadLXMERT LOAD_LXMERT]\n",
      "                             [--loadLXMERTQA LOAD_LXMERT_QA] [--fromScratch]\n",
      "                             [--mceLoss] [--llayers LLAYERS]\n",
      "                             [--xlayers XLAYERS] [--rlayers RLAYERS]\n",
      "                             [--taskMatched] [--taskMaskLM] [--taskObjPredict]\n",
      "                             [--taskQA] [--visualLosses VISUAL_LOSSES]\n",
      "                             [--qaSets QA_SETS]\n",
      "                             [--wordMaskRate WORD_MASK_RATE]\n",
      "                             [--objMaskRate OBJ_MASK_RATE] [--multiGPU]\n",
      "                             [--numWorkers NUM_WORKERS]\n",
      "ipykernel_launcher.py: error: unrecognized arguments: -f /users/cdwivedi/.local/share/jupyter/runtime/kernel-edcc6253-8fcd-4ad9-8951-5d1723147af0.json\n"
     ]
    },
    {
     "ename": "SystemExit",
     "evalue": "2",
     "output_type": "error",
     "traceback": [
      "An exception has occurred, use %tb to see the full traceback.\n",
      "\u001b[0;31mSystemExit\u001b[0m\u001b[0;31m:\u001b[0m 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/users/cdwivedi/anaconda3/envs/project/lib/python3.6/site-packages/IPython/core/interactiveshell.py:3334: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
      "  warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
     ]
    }
   ],
   "source": [
    "import tasks.vqa_model.VQAModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../../snap/vqa/vqa_lxr955_tiny/9.pth\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_dict = torch.load(path)\n",
    "model.load_state_dict(state_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'collections.OrderedDict' object has no attribute 'lxrt_encoder'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-29-1676a2946b41>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlxrt_encoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'collections.OrderedDict' object has no attribute 'lxrt_encoder'"
     ]
    }
   ],
   "source": [
    "model.lxrt_encoder.model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
